package mytf.core.crawler;

import java.util.Set;

import mytf.core.toolbox.DownAsFile;
import mytf.core.toolbox.HtmlParserTool;
import mytf.core.toolbox.LinkQ;

public class Crawler {
	
	public void crawling (String[] seeds){
		
		String seed = seeds[0];
		
		LinkQ linkQ = new LinkQ();
		linkQ.addUnVisitedUrl(seed);
		while(!linkQ.isUnVisitedUrlEmpty() && linkQ.getVisitedNum()< 10 ){
			String url = (String)linkQ.unVisitedDeQueue();
			if(url==null)continue;
			DownAsFile downfile = new DownAsFile();
			downfile.downFile(url);
			linkQ.addVisitedUrl(url);
			Set<String> links = HtmlParserTool.extracLinks(url);
			for(String strUrl:links){
				linkQ.addUnVisitedUrl(strUrl);
			}
		}
		
	}
	
	
}
